import sys

def producingNumbers():
    filePathBase = "/data5/team/weijiang/pruningProjectTrecEvaluationResults/top-10000-AND-Semantics-Pruning-PartialBM25/2004-2006/"
    
    fileNameList = []
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_100%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_90%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_80%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_70%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_60%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_50%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_40%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_30%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_20%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_10%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_5%_04-06.all.measures")
    fileNameList.append("trec_ad_hoc_results_daat_and_wei_from_pruning_index_1%_04-06.all.measures")
    
    
    for relativeFileName in fileNameList:
        absoluteFileName = filePathBase + relativeFileName
        print "fileName:",absoluteFileName
        f =file(absoluteFileName, 'r')
        for currentLine in f.readlines():
            elementsList = currentLine.strip().split("\t")
            tag = elementsList[0].strip()
            id = elementsList[1].strip()
            score = elementsList[2].strip()
            if id == "all":
                if tag == "num_q" or tag == "num_ret" or tag == "num_rel" or tag == "num_rel_ret" or tag == "P_10" or tag == "P_100" or tag == "recall_500" or tag == "recall_1000" or tag == "map":
                    #print tag,id,score
                    outputLine = score
                    #outputLine = tag + ":" + score
                    print outputLine
                    
def analyzingNumbers():
    queryResultNumberDict = {}
    for i in range(701,851):
        queryResultNumberDict[i] = 0
    
    fileName = "/data/weijiang/.temp"
    fileHandlder = open(fileName,"r")
    for line in fileHandlder.readlines():
        lineElements = line.strip().split("\t")
        #print lineElements
        if lineElements[0].strip() == "num_ret":
            #print lineElements[1],lineElements[2]
            if lineElements[1] != "all" and int(lineElements[1]) in queryResultNumberDict:
                queryResultNumberDict[ int(lineElements[1]) ] = int(lineElements[2])
            #print lineElements[2]
    queryResultNumberList = []
    for queryNumber in queryResultNumberDict:
        queryResultNumberList.append((queryNumber,queryResultNumberDict[queryNumber]))
    queryResultNumberList.sort(cmp=None, key=None, reverse=False)
    for tuple in queryResultNumberList:
        (queryNumber,numberOfResults) = tuple
        #print queryNumber,numberOfResults
        print numberOfResults
        
def analyzingNumbers2():
    fileName = "/data5/team/weijiang/pruningProjectResults/top-10000-AND-Semantics-Pruning-PartialBM25/2004-2006/trec_ad_hoc_results_daat_and_wei_from_pruning_index_10%_04-06"
    fileHandlder = open(fileName,"r")
    tempCounter = 0
    uniqueNumberCounter = 0
    tempResultsList = {}
     
    for line in fileHandlder.readlines():
        lineElements = line.strip().split("\t")
        print lineElements
        if lineElements[0] == "749":
            tempCounter += 1
            
            if lineElements[2] not in tempResultsList:
                tempResultsList[lineElements[2]] = 1
                uniqueNumberCounter += 1
            else:
                tempResultsList[lineElements[2]] += 1
            
    print "tempCounter:",tempCounter
    print "uniqueNumberCounter:",uniqueNumberCounter
    print "tempResultsList:",len(tempResultsList)

def producingIntersectionNumbers():
    print "producingIntersectionNumbers() called."
    queryAnswersGoldStandardDictAt10000 = {}
    queryAnswersComparedDictAt10000 = {}
    
    queryAnswersGoldStandardDictAt1000 = {}
    queryAnswersComparedDictAt1000 = {}
    
    queryAnswersGoldStandardDictAt100 = {}
    queryAnswersComparedDictAt100 = {}

    queryAnswersGoldStandardDictAt10 = {}
    queryAnswersComparedDictAt10 = {}    
    
    goldStandardFileName = "/data5/team/weijiang/pruningProjectResults/top-10000-OR-Semantics-Pruning-PartialBM25/2004-2006/trec_ad_hoc_results_daat_or_wei_from_pruning_index_100%_04-06"
    compareFileName = "/data5/team/weijiang/pruningProjectResults/top-10000-OR-Semantics-Pruning-PartialBM25/2004-2006/trec_ad_hoc_results_daat_or_wei_from_pruning_index_100%_04-06"

    print "100%"
    
    goldStadardFileHandler = open(goldStandardFileName,"r")
    
    
    for line in goldStadardFileHandler.readlines():
        lineElements = line.strip().split("\t")
        #print "lineElements:",lineElements
        
        # build the gold standard dict @10000
        if lineElements[0] not in queryAnswersGoldStandardDictAt10000:
            queryAnswersGoldStandardDictAt10000[ lineElements[0] ] = []
            queryAnswersGoldStandardDictAt10000[ lineElements[0] ].append( lineElements[2] )                                  
        else:
            queryAnswersGoldStandardDictAt10000[ lineElements[0] ].append( lineElements[2] )
            
        # build the gold standard dict @1000   
        if lineElements[0] not in queryAnswersGoldStandardDictAt1000 and int(lineElements[3]) >= 0 and int(lineElements[3]) <= 999:
                queryAnswersGoldStandardDictAt1000[ lineElements[0] ] = []
                queryAnswersGoldStandardDictAt1000[ lineElements[0] ].append( lineElements[2] )
        elif int(lineElements[3]) >= 0 and int(lineElements[3]) <= 999:
            queryAnswersGoldStandardDictAt1000[ lineElements[0] ].append( lineElements[2] )
                
        # build the gold standard dict @100
        if lineElements[0] not in queryAnswersGoldStandardDictAt100 and int(lineElements[3]) >= 0 and int(lineElements[3]) <= 99:
            queryAnswersGoldStandardDictAt100[ lineElements[0] ] = []
            queryAnswersGoldStandardDictAt100[ lineElements[0] ].append( lineElements[2] )
        elif int(lineElements[3]) >= 0 and int(lineElements[3]) <= 99:
            queryAnswersGoldStandardDictAt100[ lineElements[0] ].append( lineElements[2] )

        # build the gold standard dict @10
        if lineElements[0] not in queryAnswersGoldStandardDictAt10 and int(lineElements[3]) >= 0 and int(lineElements[3]) <= 9:
            queryAnswersGoldStandardDictAt10[ lineElements[0] ] = []
            queryAnswersGoldStandardDictAt10[ lineElements[0] ].append( lineElements[2] )
        elif int(lineElements[3]) >= 0 and int(lineElements[3]) <= 9:
            queryAnswersGoldStandardDictAt10[ lineElements[0] ].append( lineElements[2] )

    goldStadardFileHandler.close()
    
    comparedFilehandler = open(compareFileName,"r")
    for line in comparedFilehandler.readlines():
        lineElements = line.strip().split("\t")
        #print "lineElements:",lineElements
        
        # build the compared dict @10000
        if lineElements[0] not in queryAnswersComparedDictAt10000:
            queryAnswersComparedDictAt10000[ lineElements[0] ] = []
            queryAnswersComparedDictAt10000[ lineElements[0] ].append( lineElements[2] )
        else:
            queryAnswersComparedDictAt10000[ lineElements[0] ].append( lineElements[2] )

        # build the gold standard dict @1000   
        if lineElements[0] not in queryAnswersComparedDictAt1000 and int(lineElements[3]) >= 0 and int(lineElements[3]) <= 999:
                queryAnswersComparedDictAt1000[ lineElements[0] ] = []
                queryAnswersComparedDictAt1000[ lineElements[0] ].append( lineElements[2] )
        elif int(lineElements[3]) >= 0 and int(lineElements[3]) <= 999:
            queryAnswersComparedDictAt1000[ lineElements[0] ].append( lineElements[2] )
                
        # build the gold standard dict @100
        if lineElements[0] not in queryAnswersComparedDictAt100 and int(lineElements[3]) >= 0 and int(lineElements[3]) <= 99:
            queryAnswersComparedDictAt100[ lineElements[0] ] = []
            queryAnswersComparedDictAt100[ lineElements[0] ].append( lineElements[2] )
        elif int(lineElements[3]) >= 0 and int(lineElements[3]) <= 99:
            queryAnswersComparedDictAt100[ lineElements[0] ].append( lineElements[2] )

        # build the gold standard dict @10
        if lineElements[0] not in queryAnswersComparedDictAt10 and int(lineElements[3]) >= 0 and int(lineElements[3]) <= 9:
            queryAnswersComparedDictAt10[ lineElements[0] ] = []
            queryAnswersComparedDictAt10[ lineElements[0] ].append( lineElements[2] )
        elif int(lineElements[3]) >= 0 and int(lineElements[3]) <= 9:
            queryAnswersComparedDictAt10[ lineElements[0] ].append( lineElements[2] )  
        
    comparedFilehandler.close()
    
    print "load and sort BM25 gold standard..."
    for queryNumber in queryAnswersGoldStandardDictAt10000:
        queryAnswersGoldStandardDictAt10000[queryNumber].sort()
        queryAnswersGoldStandardDictAt1000[queryNumber].sort()
        queryAnswersGoldStandardDictAt100[queryNumber].sort()
        queryAnswersGoldStandardDictAt10[queryNumber].sort()
        
        '''
        print "queryNumber:",queryNumber,"# of results@10000:",len(queryAnswersGoldStandardDictAt10000[queryNumber])
        print "queryNumber:",queryNumber,"# of results@1000:",len(queryAnswersGoldStandardDictAt1000[queryNumber])
        print "queryNumber:",queryNumber,"# of results@100:",len(queryAnswersGoldStandardDictAt100[queryNumber])
        print "queryNumber:",queryNumber,"# of results@10:",len(queryAnswersGoldStandardDictAt10[queryNumber])
        '''
        '''
        if queryNumber == "800":
            print "queryAnswersGoldStandardDictAt10[queryNumber]:",queryAnswersGoldStandardDictAt100[queryNumber]
        '''
    print "DONE."

    print "load and sort the compared..."
    for queryNumber in queryAnswersComparedDictAt10000:
        queryAnswersComparedDictAt10000[queryNumber].sort()
        queryAnswersComparedDictAt1000[queryNumber].sort()
        queryAnswersComparedDictAt100[queryNumber].sort()
        queryAnswersComparedDictAt10[queryNumber].sort()
        
        '''
        print "queryNumber:",queryNumber,"# of results@10000:",len(queryAnswersComparedDictAt10000[queryNumber])
        print "queryNumber:",queryNumber,"# of results@1000:",len(queryAnswersComparedDictAt1000[queryNumber])
        print "queryNumber:",queryNumber,"# of results@100:",len(queryAnswersComparedDictAt100[queryNumber])
        print "queryNumber:",queryNumber,"# of results@10:",len(queryAnswersComparedDictAt10[queryNumber])
        '''
        '''
        if queryNumber == "800":
            print "queryAnswersComparedDictAt10[queryNumber]:",queryAnswersComparedDictAt10[queryNumber]
        ''' 
    print "DONE."
    
    
    # data preparation section
    # build the results @10000
    print "prepare the data for @10000"
    #print "queryNumber,","total@10000,","returned@10000,","intersect %"
    outputResultsListAt10000 = []
    for queryNumber in queryAnswersGoldStandardDictAt10000:
        #print "queryNumber:",queryNumber
        tempCounter = 0
        passCounter = 0
        if queryNumber in queryAnswersComparedDictAt10000:
            for docID in queryAnswersComparedDictAt10000[queryNumber]:
                if docID in queryAnswersGoldStandardDictAt10000[queryNumber]:
                    #print "p1"
                    tempCounter += 1
                else:
                    #print "p2"
                    passCounter += 1
        else:
            pass
        outputResultsListAt10000.append( (queryNumber, len(queryAnswersGoldStandardDictAt10000[queryNumber]), tempCounter, float(tempCounter)/float( len(queryAnswersGoldStandardDictAt10000[queryNumber]) ) ) )
    outputResultsListAt10000.sort(cmp=None, key=None, reverse=False)
    
    

        
    # build the results @1000
    print "prepare the data for @1000"
    #print "queryNumber,","total@1000,","returned@1000,","intersect %"
    outputResultsListAt1000 = []
    for queryNumber in queryAnswersGoldStandardDictAt1000:
        #print "queryNumber:",queryNumber
        tempCounter = 0
        passCounter = 0
        if queryNumber in queryAnswersComparedDictAt1000:
            for docID in queryAnswersComparedDictAt1000[queryNumber]:
                if docID in queryAnswersGoldStandardDictAt1000[queryNumber]:
                    #print "p1"
                    tempCounter += 1
                else:
                    #print "p2"
                    passCounter += 1
        else:
            pass
        outputResultsListAt1000.append( (queryNumber, len(queryAnswersGoldStandardDictAt1000[queryNumber]), tempCounter, float(tempCounter)/float( len(queryAnswersGoldStandardDictAt1000[queryNumber]) ) ) )
    outputResultsListAt1000.sort(cmp=None, key=None, reverse=False)
    
        
    # build the results @100
    print "prepare the data for @100"
    #print "queryNumber,","total@100,","returned@100,","intersect %"
    outputResultsListAt100 = []
    for queryNumber in queryAnswersGoldStandardDictAt100:
        #print "queryNumber:",queryNumber
        tempCounter = 0
        passCounter = 0
        if queryNumber in queryAnswersComparedDictAt100:
            for docID in queryAnswersComparedDictAt100[queryNumber]:
                if docID in queryAnswersGoldStandardDictAt100[queryNumber]:
                    #print "p1"
                    tempCounter += 1
                else:
                    #print "p2"
                    passCounter += 1
        else:
            pass
        outputResultsListAt100.append( (queryNumber, len(queryAnswersGoldStandardDictAt100[queryNumber]), tempCounter, float(tempCounter)/float( len(queryAnswersGoldStandardDictAt100[queryNumber]) ) ) )
    outputResultsListAt100.sort(cmp=None, key=None, reverse=False)
    
    
    # build the results @10
    print "prepare the data for @10"
    #print "queryNumber,","total@10,","returned@10,","intersect %"
    outputResultsListAt10 = []
    for queryNumber in queryAnswersGoldStandardDictAt10:
        #print "queryNumber:",queryNumber
        tempCounter = 0
        passCounter = 0
        if queryNumber in queryAnswersComparedDictAt10:
            for docID in queryAnswersComparedDictAt10[queryNumber]:
                if docID in queryAnswersGoldStandardDictAt10[queryNumber]:
                    #print "p1"
                    tempCounter += 1
                else:
                    #print "p2"
                    passCounter += 1
        else:
            pass
        outputResultsListAt10.append( (queryNumber, len(queryAnswersGoldStandardDictAt10[queryNumber]), tempCounter, float(tempCounter)/float( len(queryAnswersGoldStandardDictAt10[queryNumber]) ) ) )
    outputResultsListAt10.sort(cmp=None, key=None, reverse=False)
        
    '''
    # old version.
    # data display section.
    for tuple in outputResultsListAt10000:
        (queryNumber,lengthOfGold,lengthOfCompared,rate) = tuple
        print queryNumber,lengthOfGold,lengthOfCompared,rate
        #print queryNumber
        #print lengthOfGold
        #print lengthOfCompared
        #print rate

    for tuple in outputResultsListAt1000:
        (queryNumber,lengthOfGold,lengthOfCompared,rate) = tuple
        print queryNumber,lengthOfGold,lengthOfCompared,rate
        #print queryNumber
        #print lengthOfGold
        #print lengthOfCompared
        #print rate

    for tuple in outputResultsListAt100:
        (queryNumber,lengthOfGold,lengthOfCompared,rate) = tuple
        print queryNumber,lengthOfGold,lengthOfCompared,rate
        #print queryNumber
        #print lengthOfGold
        #print lengthOfCompared
        #print rate
        
    for tuple in outputResultsListAt10:
        (queryNumber,lengthOfGold,lengthOfCompared,rate) = tuple
        print queryNumber,lengthOfGold,lengthOfCompared,rate
        #print queryNumber
        #print lengthOfGold
        #print lengthOfCompared
        #print rate
    '''
    
    print "display format:","queryNumber,","intersect rate @10000,","# results intersect @10000,","intersect rate @1000,","# results intersect @1000,","intersect rate @100,","# results intersect @100,","intersect rate @10,","# results intersect @10"
    
    for i in range(0,148):
        #print "i:",i
        tupleFor10000 = outputResultsListAt10000[i]
        tupleFor1000  = outputResultsListAt1000[i]
        tupleFor100   = outputResultsListAt100[i]
        tupleFor10    = outputResultsListAt10[i]
        
        (queryNumberFor10000,lengthOfGoldFor10000,lengthOfComparedFor10000,rateFor10000) = tupleFor10000
        (queryNumberFor1000,lengthOfGoldFor1000,lengthOfComparedFor1000,rateFor1000) = tupleFor1000
        (queryNumberFor100,lengthOfGoldFor100,lengthOfComparedFor100,rateFor100) = tupleFor100
        (queryNumberFor10,lengthOfGoldFor10,lengthOfComparedFor10,rateFor10) = tupleFor10
        
        print queryNumberFor10000,rateFor10000,lengthOfComparedFor10000,rateFor1000,lengthOfComparedFor1000,rateFor100,lengthOfComparedFor100,rateFor10,lengthOfComparedFor10
    
    ''' 
    print "len(outputResultsListAt10):",len(outputResultsListAt10)
    print "len(outputResultsListAt100):",len(outputResultsListAt100)
    print "len(outputResultsListAt1000):",len(outputResultsListAt1000)
    print "len(outputResultsListAt10000):",len(outputResultsListAt10000)
    '''
    
# Main program begins here.
producingIntersectionNumbers()

#analyzingNumbers()

#analyzingNumbers2()
        
    

    
